import datetime
import getopt
import json
import os
import re
import shutil
import sys
from urllib.parse import urlencode

import PyPDF2 as PyPDF2
import requests
import wordcloud
import matplotlib.pyplot as plt
import pdfplumber
import jieba
import string
import pymysql
import pymysql.cursors
import argparse
from dbutils.pooled_db import PooledDB


def pdftotext(pdf_pash, txt_pash):
    text = ''
    with pdfplumber.open(pdf_pash) as pdf:
        for page in pdf.pages:
            text = text + page.extract_text()
    # Save the text to a text file
    with open(txt_pash, 'w', encoding='utf-8') as file:
        text = text.encode('UTF-8', 'ignore').decode('UTF-8')
        text = text.encode('utf-8', errors='replace').decode()
        file.write(text)
    return 1


def postForR(url, params):
    headers = {"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", 'Connection': 'close'}
    payload = urlencode(params)
    response = requests.post(url=url, data=payload, headers=headers)
    if response.status_code == 200:
        return response.json()


def downloadPdf(uri, url_pash):
    url = 'http://static.cninfo.com.cn/' + uri
    response = requests.get(url)
    if response.status_code == 200:
        with open(url_pash, 'wb') as file:
            file.write(response.content)


def timestamp_to_date(timestamp):
    return datetime.datetime.fromtimestamp(timestamp / 1000.0)


#这里配置数据库地址 目前所有用的为mysql
class MysqlPool:
    config = {
        'creator': pymysql,
        'host': '192.168.199.221',#mysql IP地址
        'port': 3306,#mysql 端口
        'user': 'root',#mysql 账户
        'password': '083d65e135eba065',#mysql 密码
        'db': 'jinrong',#mysql 所属库
        'charset': 'utf8',#使用的编码
        'maxconnections': 70,  # 连接池最大连接数量
        'cursorclass': pymysql.cursors.DictCursor
    }
    # 借助连接池连接数据库
    pool = PooledDB(**config)

    def __enter__(self):
        self.conn = MysqlPool.pool.connection()
        self.cursor = self.conn.cursor()
        return self

    def __exit__(self, type, value, trace):
        self.cursor.close()
        self.conn.close()


# 执行函数
def execute(database_commands, s_value=None, i=0):
    """

    :param database_commands: 数据库的执行语句
    :param s_value: %s代表的变量名称
    :param i: 执行判断符，当i为1时执行状态判断 ,默认值为0
    """
    # 捕获异常，并报错
    try:
        with MysqlPool() as db:
            # 执行操作命令 database_commands:数据库的查询语句 s_value:当查询语句出现%s时%s对应的值，默认为None
            db.cursor.execute(database_commands, s_value)
            # 向数据库提交命令
            db.conn.commit()
            # 当执行查询操作时打印并返回查询的数据
            if "select" in database_commands and i == 0:
                # 打印查询的全部数据,fetchall()的返回值为list格式的字典集
                # print(db.cursor.fetchall())
                # 返回查询的全部数据
                return db.cursor.fetchall()
            elif i == 1:
                return db.cursor.fetchall()
            # o(*￣︶￣*)o此处调用__exit__()
    except Exception as e:
        print(e)


def totalTojson(txt_path, id):
    content = getText(txt_path)
    jieba.load_userdict("./userdict.txt")
    words = jieba.cut(content)
    json_str = ",".join(words)
    counts = {}
    array_str = json_str.split(',')
    in_array = ["EB级存储", "NFC支付", "差分隐私技术", "大数据", "第三方支付", "多方安全计算", "分布式计算",
                "股权众筹融资", "互联网金融", "机器学习", "开放银行", "类脑计算", "量化金融", "流计算", "绿色计算",
                "内存计算", "区块链", "人工智能", "认知计算", "融合架构", "商业智能", "身份验证", "深度学习",
                "生物识别技术", "数据可视化", "数据挖掘", "数字货币", "投资决策辅助系统", "图计算", "图像理解", "网联",
                "文本挖掘", "物联网", "信息物理系统", "虚拟现实", "移动互联", "移动支付", "亿级并发", "异构数据",
                "语义搜索", "语音识别", "云计算", "征信", "智能金融合约", "智能客服", "智能数据分析", "智能投顾",
                "自然语言处理"]
    total_num = 0
    for word in array_str:
        rword = ""
        if word in in_array:
            rword = word
        if rword:
            total_num += 1
            counts[rword] = counts.get(rword, 0) + 1

    items = list(counts.items())
    items.sort(key=lambda x: x[1], reverse=True)
    total_json = json.dumps(items, ensure_ascii=False)
    updateTotalNum(total_num, id)
    return total_json


def updateTotalNum(total_num, id):
    sql_update = 'update attach set total_num=%s where id=%s'
    value = (total_num, id)
    execute(sql_update, value)


def getText(txt_path):
    txt = open(txt_path, "r", encoding="utf-8").read()

    # 使用空格个来替换掉所有的符号
    for ch in "!\"#$%&()*+,-./:;<>=?@[\\]_'{|}~[。][，]|\s|\t|\n|[：][；][、][ ][（][）][△][㈡][㈠][⑴][⑶][％][”][《][》][－][“][①][②][③][④][⑤][⑦][⑧][⑵][⑥][㈤][⑷][⑹][⑼]":
        txt = txt.replace(ch, '')
        txt = re.sub(r'\d+|\s|\t|\n|[—]|[〇]', '', txt)
    return txt


def fileMove(path, topath, file_name):
    pathinfo = "D:\\MobileHardDisk\\python\\jinrong\\filepash"
    path = pathinfo + "\\" + path + "\\" + file_name
    topath = pathinfo + "\\" + topath + "\\" + file_name
    shutil.move(path, topath)


class ContentOcr():
    def __init__(self):
        global opts
        super(ContentOcr, self).__init__()
        parser = argparse.ArgumentParser(description=None)
        parser.add_argument('-p', '--page', type=int)
        parser.add_argument('-a', '--action', required=True)
        args = parser.parse_args()
        action = args.action
        pageNum = args.page
        match action:
            case "getData":
                # 第一步 先去www.cninfo.com.cn 这个网站 获取年报的所有数据并存入库
                self.getRemotelyDataAll()
            case "downloadPdf":
                # 第二步 下载PDF文件
                if None == pageNum:
                    pageNum = 0
                pageSize = 100
                self.downloadPdf(pageNum, pageSize)
            case "dataToText":
                # 第三步 PDF文件转txt文件
                if None == pageNum:
                    pageNum = 0
                pageSize = 1000
                self.dataToText(pageNum, pageSize)
            case "analyzeWords":
                # 第四步 读取txt文件分析词语
                if None == pageNum:
                    pageNum = 0
                pageSize = 1000
                self.txtAnalyzeWords(pageNum, pageSize)
            case _:
                pass

    def getRemotelyDataAll(self):
        #需要请求的地址URL
        url = "http://www.cninfo.com.cn/new/hisAnnouncement/query"
        #从多久开始
        nextTime = "2011-01-01"
        #拿到当天为止
        today = datetime.datetime.now().date()
        #循环了多少次
        dayNumber = 0
        while True:
            #每次获取100条
            totalPages = 101
            lastTime = nextTime
            lastTimeDate = datetime.datetime.strptime(lastTime, '%Y-%m-%d').date()
            #每次加一天
            nextTimeData = lastTimeDate + datetime.timedelta(days=1)
            nextTime = str(nextTimeData)
            for i in range(1, totalPages):
                #请求接口需要的参数
                params = {
                    "pageNum": i,
                    "pageSize": "30",
                    "column": "szse",
                    "tabName": "fulltext",
                    "plate": "",
                    "stock": "",
                    "searchkey": "",
                    "secid": "",
                    "category": "category_ndbg_szsh",
                    "trade": "",
                    "seDate": lastTime + "~" + nextTime,
                    "sortName": "",
                    "sortType": "",
                    "isHLtitle": "true"
                }
                #post请求
                response = postForR(url, params)
                if None == response["announcements"]:
                    print(lastTime + "~" + nextTime + ":未获取到数据")
                    break
                for item in response["announcements"]:
                    if '英文版' in item['shortTitle']:
                        break
                    if '已取消' in item['shortTitle']:
                        break
                    dataInfo = self.getDataByCodeAndAttachPath(item['secCode'], item['adjunctUrl'])
                    if dataInfo:
                        break
                    if item['secCode']:
                        announcementDate = timestamp_to_date(item['announcementTime'])
                        self.insertAttach(item['secCode'], item['secName'], item['shortTitle'], item['adjunctUrl'],
                                          announcementDate)
            dayNumber += 1
            if (today == nextTimeData):
                print(dayNumber)
                break

    def downloadPdf(self, pageNum, pageSize):
        while True:
            pages = pageNum * pageSize
            print("start_limit:" + str(pageNum))
            print("pages:" + str(pages))
            dataJson = execute(
                "select * from attach where attach_path <> '' and dw_path is null ORDER BY id asc LIMIT "
                + str(pages) + "," + str(pageSize))
            if not dataJson:
                break
            for datas in dataJson:
                if datas['attach_path']:
                    char_to_remove = "*"
                    if not datas["title"]:
                        datas["title"] = datas["name"]
                    datas["title"] = datas["title"].replace(char_to_remove, "")
                    pdf_file_name = str(datas["id"]) + "_" + datas["title"] + '.pdf'
                    url_pash = './filepash/pdf/' + pdf_file_name
                    try:
                        downloadPdf(datas["attach_path"], url_pash)
                        self.updateFilePath(url_pash, datas["id"])
                        print(datas["id"])
                    except Exception as e:
                        print(e)
                        continue

    def dataToText(self, pageNum, pageSize):
        while True:
            pages = pageNum * pageSize
            print("start_limit:" + str(pageNum))
            print("pages:" + str(pages))
            dataJson = execute(
                "select * from attach where dw_path <> '' and txt_path is null ORDER BY id asc LIMIT "
                + str(pages) + "," + str(pageSize))
            if not dataJson:
                break
            for datas in dataJson:
                if datas['dw_path']:
                    text_file_name = str(datas["id"]) + "_" + datas["title"] + '.txt'
                    text_pash = './filepash/text/' + text_file_name
                    try:
                        if os.path.exists(text_pash):  # 判断文件是否存在
                            self.updateTxtFilePath(text_pash, datas["id"])
                            print("直接更新")
                        else:
                            result = pdftotext(datas['dw_path'], text_pash)
                            if 1 == result:
                                self.updateTxtFilePath(text_pash, datas["id"])
                                print("转义更新")
                        print(datas["id"])
                    except Exception as e:
                        print(datas["id"])
                        print(e)
                        continue

    def txtAnalyzeWords(self, pageNum, pageSize):
        while True:
            pages = pageNum * pageSize
            print("start_limit:" + str(pageNum))
            print("pages:" + str(pages))
            dataJson = execute(
                "select * from attach where txt_path <> '' and total_json is null ORDER BY id asc LIMIT "
                + str(pages) + "," + str(pageSize))
            if not dataJson:
                break
            for datas in dataJson:
                if datas['txt_path']:
                    try:
                        total_json = totalTojson(datas['txt_path'], datas["id"])
                        self.updateTotalJson(total_json, datas["id"])
                        print(datas["id"])
                    except Exception as e:
                        print(datas["id"])
                        print(e)
                        continue

    def updateFilePath(self, url_path, id):
        sql_update = 'update attach set dw_path=%s where id=%s'
        value = (url_path, id)
        execute(sql_update, value)

    def updateTxtFilePath(self, url_path, id):
        sql_update = 'update attach set txt_path=%s where id=%s'
        value = (url_path, id)
        execute(sql_update, value)

    def updateTotalJson(self, total_json, id):
        sql_update = 'update attach set total_json=%s where id=%s'
        value = (total_json, id)
        execute(sql_update, value)

    def getDataByCodeAndAttachPath(self, code, attachPath):
        sql_select = "select * from attach where code=%s and attach_path=%s ORDER BY id asc"
        value = (code, attachPath)
        return execute(sql_select, value)

    def insertAttach(self, code, name, title, attachPath, publishTime):
        sql_insert = "INSERT INTO attach (code,name,title,attach_path,publish_time) VALUES (%s,%s,%s,%s,%s)"
        value = (code, name, title, attachPath, publishTime)
        execute(sql_insert, value)


if __name__ == '__main__':
    main = ContentOcr()
